package featureSelect;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;

import tool.MySentiDemo;
import mlProject.DocModel;
import mlProject.DocModelReader;
import mlProject.DocModel.Feature;

/*
 * 2.1  统计正负分类的文档数:N1、N2。
 * 2.2  统计每个词的正文档出现频率（A）、负文档出现频率（B）、正文档不出现频率(C)、负文档不出现频率(D)。
 * 2.3  计算信息熵
 */
public class InfoGainWithSentiment {
	static Comparator<Entry<String, Double>> comparator_map = new Comparator<Entry<String, Double>>() {
		public int compare(Entry<String, Double> e1, Entry<String, Double> e2) {
			return e2.getValue().compareTo(e1.getValue());// 降序
		}
	};

	public static class WordFrequency {
		public int A;
		public int B;
		public int C;
		public int D;

		public WordFrequency() {
			A = 0;
			B = 0;
			C = 0;
			D = 0;
		}
		public WordFrequency(int a,int b,int c,int d) {
			A = a;
			B = b;
			C = c;
			D = d;
		}
	}

	public static Map<String, Double> computeIG(String filename) {
		int N1 = 0;
		int N2 = 0;
		double entropy = 0.0;
		Map<String, Double> infoIG = new HashMap<String, Double>();
		ArrayList<DocModel> docs = DocModelReader.readFromFile(filename);
		Map<String, WordFrequency> wordFrequencyMap = new HashMap<String, WordFrequency>();
		for (int i = 0; i < docs.size(); i++) {
			DocModel doc = docs.get(i);
			if (doc.label.equals("positive"))
				N1++;
			else
				N2++;
		}
		for (int i = 0; i < docs.size(); i++) {
			DocModel doc = docs.get(i);
			ArrayList<Feature> features = doc.features;
			for (int j = 0; j < features.size(); j++) {
				if (!wordFrequencyMap.containsKey(features.get(j).name))
					wordFrequencyMap.put(features.get(j).name, new WordFrequency(0,0,N1,N2));
			}
		}
		/*List<Map.Entry<String,WordFrequency>> list=new ArrayList<Map.Entry<String,WordFrequency>>(wordFrequencyMap.entrySet());
		Map<String,Integer> list_index=new HashMap<String,Integer>();
		for (Integer j=0;j<list.size();j++) {
			list_index.put(list.get(j).getKey(), j);
		}*/
		for (int i = 0; i < docs.size(); i++) {
			//System.out.println(i);
			DocModel doc = docs.get(i);
			ArrayList<Feature> features = doc.features;
			for (int j = 0; j < features.size(); j++) {
				String key=features.get(j).name;
				if (doc.label.equals("positive")) {
					wordFrequencyMap.get(key).A++;
					wordFrequencyMap.get(key).C--;
				} else {
					wordFrequencyMap.get(key).B++;
					wordFrequencyMap.get(key).D--;
				}
			}
		}
		Double p1 = N1 * 1.0 / (N1 + N2);
		Double p2 = N2 * 1.0 / (N1 + N2);
		entropy = -(p1 * logBase2(p1) + p2 * logBase2(p2));
		Double pAB = 0.0;
		Double pCD = 0.0;
		Double pA = 0.0;
		Double pB = 0.0;
		Double pC = 0.0;
		Double pD = 0.0;

		for (Map.Entry<String, WordFrequency> it : wordFrequencyMap.entrySet()) {
			pAB = (it.getValue().A + it.getValue().B) * 1.0 / (N1 + N2);
			pCD = (it.getValue().C + it.getValue().D) * 1.0 / (N1 + N2);
			pA = it.getValue().A * 1.0 / (it.getValue().A + it.getValue().B);
			pB = it.getValue().B * 1.0 / (it.getValue().A + it.getValue().B);
			pC = it.getValue().C * 1.0 / (it.getValue().C + it.getValue().D);
			pD = it.getValue().D * 1.0 / (it.getValue().C + it.getValue().D);
			Double igTemp = entropy + pAB * (pA * logBase2(pA + 0.000000001) + pB * logBase2(pB + 0.000000001))
					+ pCD * (pC * logBase2(pC) + pD * logBase2(pD));
			infoIG.put(it.getKey(), igTemp);
		}
		// List<Map.Entry<String,Double>> list=new
		// ArrayList<Map.Entry<String,Double>>(infoIG.entrySet());
		// Collections.sort(list, comparator_map);
		return infoIG;
		/*
		 * ArrayList<String> featureSelected=new ArrayList<String>(); for(int
		 * i=0;i<numOfFeature;i++){ featureSelected.add(list.get(i).getKey()); }
		 * return featureSelected;
		 */
	}

	public static Map<String, Double> computeMixedIG() {
		Map<String, Double> wholeIG = computeIG("newTrain.txt");
		Map<String, Double> bookIG = computeIG("book.txt");
		Map<String, Double> DVDIG = computeIG("DVD.txt");
		Map<String, Double> electronicIG = computeIG("electronic.txt");
		Map<String, Double> kitchenIG = computeIG("kitchen.txt");
		Map<String, Double> mixedIG = new HashMap<String, Double>();
		/*int count=0;
		System.out.println("mapsize:"+wholeIG.size());
		for (Map.Entry<String, Double> it : wholeIG.entrySet()) {
			String key = it.getKey();System.out.println(count);count++;
			Double mixValue = 0.0;
			mixValue += wholeIG.get(key);
			if(bookIG.containsKey(key))
				mixValue += bookIG.get(key);
			if(DVDIG.containsKey(key))
				mixValue += DVDIG.get(key);
			if(electronicIG.containsKey(key))
				mixValue += electronicIG.get(key);
			if(kitchenIG.containsKey(kitchenIG))
				mixValue += kitchenIG.get(key);
			mixedIG.put(key, mixValue);
		}*/
		String key="";
		for (Map.Entry<String, Double> it : bookIG.entrySet()) {
			key=it.getKey();
			if(wholeIG.containsKey(key)) wholeIG.put(key, wholeIG.get(key)+it.getValue());
		}
		for (Map.Entry<String, Double> it : DVDIG.entrySet()) {
			key=it.getKey();
			if(wholeIG.containsKey(key)) wholeIG.put(key, wholeIG.get(key)+it.getValue());
		}
		for (Map.Entry<String, Double> it : electronicIG.entrySet()) {
			key=it.getKey();
			if(wholeIG.containsKey(key)) wholeIG.put(key, wholeIG.get(key)+it.getValue());
		}
		for (Map.Entry<String, Double> it : kitchenIG.entrySet()) {
			key=it.getKey();
			if(wholeIG.containsKey(key)) wholeIG.put(key, wholeIG.get(key)+it.getValue());
		}
		return wholeIG;
	}

	public static Map<String, Double> addSentiEffect(double effect) {
		String sentiFilePath = "src/main/resources/" + "SentiWordNet_3.0.0_20130122.txt";
		MySentiDemo sentiwordnet;
		Map<String, Double> finalIG = new HashMap<String, Double>();
		try {
			sentiwordnet = new MySentiDemo(sentiFilePath);
			Map<String, Double> mixedIG = computeMixedIG();
			for (Map.Entry<String, Double> it : mixedIG.entrySet()) {
				String key = it.getKey();
				Double value = it.getValue();
				String[] keySplit = key.split("_");
				Double max = -100.0;
				Double score = 0.0;
				for (int i = 0; i < keySplit.length; i++) {
					score = sentiwordnet.getSentiScore(keySplit[i]);
					if (score > max)
						max = score;
				}
				value *= (effect * max + 1);
				finalIG.put(key, value);
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return finalIG;
	}

	public static ArrayList<String> getFeatureSelected(int numOfFeature) {
		Map<String, Double> IG = addSentiEffect(0.2);
		List<Map.Entry<String, Double>> list = new ArrayList<Map.Entry<String, Double>>(IG.entrySet());
		Collections.sort(list, comparator_map);
		ArrayList<String> featureSeleted = new ArrayList<String>();
		for (int i = 0; i < numOfFeature; i++) {
			featureSeleted.add(list.get(i).getKey());
		}
		return featureSeleted;
	}
	public static ArrayList<String> getFeatureSelectedWholeIG(int numOfFeature) {
		Map<String, Double> IG =  computeIG("newTrain.txt");
		List<Map.Entry<String, Double>> list = new ArrayList<Map.Entry<String, Double>>(IG.entrySet());
		Collections.sort(list, comparator_map);
		ArrayList<String> featureSeleted = new ArrayList<String>();
		for (int i = 0; i < numOfFeature; i++) {
			featureSeleted.add(list.get(i).getKey());
		}
		return featureSeleted;
	}
	public static ArrayList<String> getFeatureSelectedMixedIG(int numOfFeature) {
		Map<String, Double> IG =  computeMixedIG();
		List<Map.Entry<String, Double>> list = new ArrayList<Map.Entry<String, Double>>(IG.entrySet());
		Collections.sort(list, comparator_map);
		ArrayList<String> featureSeleted = new ArrayList<String>();
		for (int i = 0; i < numOfFeature; i++) {
			featureSeleted.add(list.get(i).getKey());
		}
		return featureSeleted;
	}
	public static Double logBase2(Double x) {
		return Math.log(x) / Math.log(2);
	}

	public static void main(String[] args) {
		//File file_out = new File("src/main/resources/featureWithIGAndSentiment.txt");
		//File file_out = new File("src/main/resources/featureWithWholeIG.txt");
		File file_out = new File("src/main/resources/featureWithMixedIG.txt");
		try {
			BufferedWriter bw_out = new BufferedWriter(new FileWriter(file_out, false));

			//ArrayList<String> featureSeleted = getFeatureSelected();
			//ArrayList<String> featureSeleted = getFeatureSelectedWholeIG();
			ArrayList<String> featureSeleted = getFeatureSelectedMixedIG(600);
			for (int i = 0; i < featureSeleted.size(); i++) {
				bw_out.write(featureSeleted.get(i));
				bw_out.newLine();
			}
			bw_out.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}
